'''
A tool for downloading & decoding the land usage shape files for an airport project using the Terragear tools.
The DOCUMENTATION constant contains a full user documentation.

Licensed under the GPL.

@author: screwdriver
'''

'''
Version history:
----------------
2009-03-21 screwdirver Created.
2009-03-21 screwdirver Implemented command line argument - project path.
2009-03-21 screwdirver Implemented command line option - configuration file.
2009-03-21 screwdirver Implemented command line option - Terragear root.
2009-03-21 screwdirver Implemented configuration file parsing.
2009-03-21 screwdirver Implemented project parsing & area detection.
2009-03-21 screwdirver Implemented downloading.
2009-03-21 screwdirver Implemented shape file unzipping.
2009-03-23 screwdirver Implemented shape-decode invoking.
2009-03-23 screwdirver Implemented default ShapeConfig.xml spawning.
2009-03-23 screwdirver Released version 1.0.0
---
2009-03-27 screwdriver Refactoring shapeconfig - it now offers a factory.
2009-03-27 screwdriver Put debug macro into it's own module.
'''

import os;
import os.path;
import io;
import fnmatch;
import subprocess;
import string;

from debug import debug;

from string import Template;
from optparse import OptionParser;
from downloader import Downlaoder;
from shapeprocessorexceptions import DownloadShapesException;
from shapeconfig import getShapeConfigFactory, ShapeConfigFactory, ShapeConfig, ShapeFile, Shape, Line, Point;
from subprocess import CalledProcessError;

MAJOR_VERSION = 1;
MINOR_VERSION = 0;
MAINTENANCE = 0;

VERSION = [MAJOR_VERSION,".", MINOR_VERSION,".", MAINTENANCE];

USAGE = "Usage: %prog [options] project";

CONFIG_OPTION_USAGE = "Specifies the configuration file.";
TERRAGEAR_OPTION_USAGE = "The root directory of the Terragear installation.";

MISSING_PROJECT = "Please specify a project path in the argument.";
BAD_CONFIG_FILE = "Bad configuration file \"$configurationFile\".";
BAD_PROJECT_PATH = "Bad project path \"$projectPath\".";
PROJECT_NOT_A_DIRECTORY = "\"$project\" is not a directory.";
MISSING_TERRAGEAR_ROOT = "Terragear root directory \"$directory\" does not exist.";
MISSING_UNZIP = "Missing unzip utility in \"$directory\".";
MISSING_SHAPE_DECODE = "Missing shape-decode utility in \"$directory\".";
SHAPE_DECODE_ERROR = "shape-decode returned with error-code $code";

GENAPTS_OUTPUT_NOT_FOUND = "The output of gentaps was not found \"$project\".";

SHAPE_CONFIG_SCHEMA = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<xsd:schema xmlns:ecore="http://www.eclipse.org/emf/2002/Ecore" xmlns:shapeconfig="http://shapeconfig/1.0" xmlns:xsd="http://www.w3.org/2001/XMLSchema" ecore:nsPrefix="shapeconfig" ecore:package="shapeconfig" targetNamespace="http://shapeconfig/1.0">
  <xsd:import namespace="http://www.eclipse.org/emf/2002/Ecore" schemaLocation="platform:/plugin/org.eclipse.emf.ecore/model/Ecore.xsd"/>
  <xsd:element ecore:ignore="true" name="ShapeFile" type="shapeconfig:ShapeFile"/>
  <xsd:element ecore:ignore="true" name="ShapeConfig" type="shapeconfig:ShapeConfig"/>
  <xsd:element ecore:ignore="true" name="ShapeType" type="shapeconfig:ShapeType"/>
  <xsd:element ecore:ignore="true" name="Line" type="shapeconfig:Line"/>
  <xsd:element ecore:ignore="true" name="Shape" type="shapeconfig:Shape"/>
  <xsd:element ecore:ignore="true" name="Point" type="shapeconfig:Point"/>
  <xsd:complexType name="ShapeFile">
    <xsd:sequence>
      <xsd:element ecore:resolveProxies="true" name="shapeType" type="shapeconfig:ShapeType"/>
    </xsd:sequence>
    <xsd:attribute name="name" type="ecore:EString" use="required"/>
    <xsd:attribute name="material" type="ecore:EString" use="required"/>
  </xsd:complexType>
  <xsd:complexType name="ShapeConfig">
    <xsd:sequence>
      <xsd:element ecore:resolveProxies="true" maxOccurs="unbounded" minOccurs="0" name="shapeFile" type="shapeconfig:ShapeFile"/>
    </xsd:sequence>
  </xsd:complexType>
  <xsd:complexType abstract="true" ecore:interface="true" name="ShapeType"/>
  <xsd:complexType name="Line">
    <xsd:complexContent>
      <xsd:extension base="shapeconfig:Shape">
        <xsd:attribute ecore:unsettable="false" name="width" type="ecore:EDouble" use="required"/>
      </xsd:extension>
    </xsd:complexContent>
  </xsd:complexType>
  <xsd:complexType name="Shape">
    <xsd:complexContent>
      <xsd:extension base="shapeconfig:ShapeType">
        <xsd:attribute ecore:unsettable="false" name="maxSegmentLength" type="ecore:EDouble" use="required"/>
      </xsd:extension>
    </xsd:complexContent>
  </xsd:complexType>
  <xsd:complexType name="Point">
    <xsd:complexContent>
      <xsd:extension base="shapeconfig:ShapeType">
        <xsd:attribute ecore:unsettable="false" name="size" type="ecore:EDouble" use="required"/>
      </xsd:extension>
    </xsd:complexContent>
  </xsd:complexType>
</xsd:schema>""";

DEFAULT_SHAPE_CONFIG_XML = """<?xml version="1.0" encoding="UTF-8"?>
<shapeconfig:ShapeConfig xmlns:ecore="http://www.eclipse.org/emf/2002/Ecore" xmlns:shapeconfig="http://shapeconfig/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://shapeconfig/1.0 ShapeConfig.xsd ">
	<shapeFile name="v0_landmass" material="Default" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_floodland" material="Default" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_sand" material="Sand" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_lava" material="Lava" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_glacier" material="Glacier" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_packice" material="PackIce" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_polarice" material="PolarIce" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_littoral" material="Littoral" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_barrencover" material="BarrenCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_grasscover" material="GrassCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_scrubcover" material="ScrubCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_marsh" material="Marsh" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_bog" material="Bog" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_herbtundracover" material="HerbTundraCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_mixedcroppasturecover" material="MixedCropPastureCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_drycroppasturecover" material="DryCropPastureCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_irrcroppasturecover" material="IrrCropPastureCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_deciduousbroadcover" material="DeciduousBroadCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_evergreenbroadcover" material="EvergreenNeedleCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_mixedforestcover" material="MixedForestCover" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_lake" material="Lake" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_intermittentlake" material="IntermittentLake" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_urban" material="Urban" >
		<shapeType xsi:type="shapeconfig:Shape" maxSegmentLength="100" />
	</shapeFile>
	<shapeFile name="v0_stream" material="Stream" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="30" />
	</shapeFile>
	<shapeFile name="v0_intermittentstream" material="IntermittentStream" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="10" />
	</shapeFile>
	<shapeFile name="v0_canal" material="Canal" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="20" />
	</shapeFile>
	<shapeFile name="v0_road" material="Road" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="5" />
	</shapeFile>
	<shapeFile name="v0_freeway" material="Freeway" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="25" />
	</shapeFile>
	<shapeFile name="v0_railroad1" material="Railroad" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="3" />
	</shapeFile>
	<shapeFile name="v0_railroad2" material="Railroad" >
		<shapeType xsi:type="shapeconfig:Line" maxSegmentLength="50" width="6" />
	</shapeFile>
	<shapeFile name="v0_town" material="Town" >
		<shapeType xsi:type="shapeconfig:Point" size="2000" />
	</shapeFile>
</shapeconfig:ShapeConfig>""";


downloaders = [];


def __createDownloader(url, savePath):
	
	'''
	creates a downloader-thread with the given url and path and
	adds the downloader to a global list. So the main-thread can
	wait for all threads to be finished.
	'''
	
	global downloaders;
	
	if (downloaders == None):
		downloaders = [];
	downloader = Downlaoder(url, savePath);
	downloaders.append(downloader);
	downloader.start();
	
def __waitForDownloads():
	
	global downloaders;
	
	sucessfull = True;
	errors = [];	
	
	for downloader in downloaders:
		downloader.join();
		if (not downloader.sucessfull):
			sucessfull = False;
			errors.append(downloader.lasterror);
	
	if (not sucessfull):
		raise DownloadShapesException(string.join(errors, "\n"));
	
	return;

def __testFileAccess(testFile):
	
	try:
		with io.open(testFile, "a+b") as file:
			file.close();
		os.remove(testFile);
	except Exception, e:
		raise DownloadShapesException(repr(e));
	
	return;

def __testSubDirectoryAccess(testDirectory):
	
	try:
		if (not os.path.exists(testDirectory)):
			os.mkdir(testDirectory);
			
		os.chdir(testDirectory);
		os.chdir("..");
		os.rmdir(testDirectory);
	except Exception, e:
		raise DownloadShapesException(repr(e));
	
	return;

def __testDirectory(projectDirectory):
	
	if (not os.path.exists(projectDirectory)):
		os.mkdir(projectDirectory);
	else :
		if (not os.path.isdir(projectDirectory)):
			raise DownloadShapesException(Template(PROJECT_NOT_A_DIRECTORY).safe_substitute(project=projectDirectory));
	
	currentDirectory = os.getcwd();
	
	''' ensure directory access '''
	os.chdir(projectDirectory);
	
	''' ensure directory creation and sub-directory access'''
	__testSubDirectoryAccess("testDirectory");
	
	''' ensure file access (read, write, delete)''' 
	__testFileAccess("testFile");
	
	os.chdir(currentDirectory);

	return;

def __spawnFile(file, text):
	
	with io.open(file, "wb") as file:
		file.write(text);
		file.flush();
		file.close();
	
	return;

def __spawnDefaultShapeConfigFile(configurationFile):
	
	__spawnFile(configurationFile, DEFAULT_SHAPE_CONFIG_XML);
			
	return;

def __spawnShapeConfigSchema(schemaPath):
	
	__spawnFile(schemaPath + "/ShapeConfig.xsd", SHAPE_CONFIG_SCHEMA);
	
	return;

def __checkConfigFile(configurationFile):
	
	if (not os.path.exists(configurationFile)):
		__spawnDefaultShapeConfigFile(configurationFile);
		
		schemaPath = os.path.dirname(configurationFile);
		__spawnShapeConfigSchema(schemaPath);
	
	return;

def __checkArguments(projectDirectory, configurationFile, terragearRoot):
	
	if (projectDirectory == None):
		raise DownloadShapesException(MISSING_PROJECT);
		
	try:
		__testDirectory(projectDirectory);
	except Exception, e:
		raise DownloadShapesException(Template(BAD_PROJECT_PATH).safe_substitute(projectPath=projectDirectory) + "\n" + e.message);

	
	shapeFilesDirectory = projectDirectory + "/shapefiles";
	if (not os.path.exists(shapeFilesDirectory)):
		try:
			os.mkdir(shapeFilesDirectory);
		except Exception, e:
			raise DownloadShapesException(Template(BAD_PROJECT_PATH).safe_substitute(projectPath=projectDirectory) + "\n" + repr(e));
		
	
	if (configurationFile == None):
		raise DownloadShapesException(MISSING_CONFIGURATION_FILE)
	
	try:
		__checkConfigFile(configurationFile);
	except Exception, e:
		raise DownloadShapesException(Template(BAD_CONFIG_FILE).safe_substitute(configurationFile=configurationFile) + "\n" + e.message);
	
	if (not os.path.exists(terragearRoot)):
		raise DownloadShapesException(Template(MISSING_TERRAGEAR_ROOT).safe_substitute(directory=terragearRoot));
	
	if (not os.path.exists(terragearRoot + "/unzip") and not os.path.exists(terragearRoot + "/unzip.exe")):
		raise DownloadShapesException(Template(MISSING_UNZIP).safe_substitute(directory=terragearRoot));
	
	if (not os.path.isfile(terragearRoot + "/unzip") and not os.path.isfile(terragearRoot + "/unzip.exe")):
		raise DownloadShapesException(Template(MISSING_UNZIP).safe_substitute(directory=terragearRoot));
	
	if (not os.path.exists(terragearRoot + "/shape-decode") and not os.path.exists(terragearRoot + "/shape-decode.exe")):
		raise DownloadShapesException(Template(MISSING_SHAPE_DECODE).safe_substitute(directory=terragearRoot));

	if (not os.path.isfile(terragearRoot + "/shape-decode") and not os.path.isfile(terragearRoot + "/shape-decode.exe")):
		raise DownloadShapesException(Template(MISSING_SHAPE_DECODE).safe_substitute(directory=terragearRoot));
	
	return;

def __createConfigurationModel(configurationFile):
	
	configuration = getShapeConfigFactory().load(configurationFile);
	
	return configuration;

def __parseTileLocation(directory):
	
	tokens = (directory[0], directory[1:4], directory[4], directory[5:7]);
	
	xmin = int(tokens[1]);
	ymax = int(tokens[3]);
	
	longitude = tokens[0];
	lattitude = tokens[2];
	
	return (xmin, ymax, longitude, lattitude);

def __retrieveArea(projectDirectory):
	
	''' we need AirportObj/<section>/<tile>/<project>.btg.gz'''
	
	airportObjects = projectDirectory + "/AirportObj";
	
	''' to prevent io exceptions later'''
	if (not os.path.exists(airportObjects) and not os.path.isdir(airportObjects)):
		raise DownloadShapesException(Template(GENAPTS_OUTPUT_NOT_FOUND).safe_substitute(project=projectDirectory))
	
	foundGenaptsOutput = False;
	
	for root, dirs, files in os.walk(airportObjects):
		for file in files:
			if (file == projectDirectory + ".btg.gz"):
				
				directory = os.path.basename(root);
				
				(xmin, ymax, longitude, lattitude) = __parseTileLocation(directory);
				
				if (longitude == 'w'):
					xmin = -xmin;
				if (lattitude == 's'):
					ymax = -ymax;
					
				xmax = xmin + 1;
				ymin = ymax -1;
				
				return (xmin, xmax, ymin, ymax);
	
	
	raise DownloadShapesException(Template(GENAPTS_OUTPUT_NOT_FOUND).safe_substitute(project=projectDirectory));
	

def __startAreaDownload(projectDirectory, layer, xmin, xmax, ymin, ymax):
	urlTemplate = Template("http://mapserver.flightgear.org/dlaction.psp?ymax=$ymax&xmin=$xmin&xmax=$xmax&ymin=$ymin&pgislayer=$layer&submit=Downlaod");
	urlString = urlTemplate.substitute(ymax=ymax, xmin=xmin, xmax=xmax, ymin=ymin, layer=layer);
	shapeFile = Template("$projectDirectory/shapefiles/$layer.zip").substitute(projectDirectory=projectDirectory, layer=layer);
	__createDownloader(urlString, shapeFile);


def _downloadShapes(projectDirectory, configuration):

	(xmin, xmax, ymin, ymax) = __retrieveArea(projectDirectory);
	
	for shapeFile in configuration.shapeFiles:
		
		layer = shapeFile.name;
		
		__startAreaDownload(projectDirectory, layer, xmin, xmax, ymin, ymax);
		
	debug("Waiting for downloads...");
	
	__waitForDownloads();
	
	debug("Downloads done.");
	
	return (xmin, xmax, ymin, ymax);
	
def _unzipShapes(projectDirectory, configuration, terragearRoot):
	
	sucessfull = True;
	
	errors = [];
	
	for shapeFile in configuration.shapeFiles:
		name = shapeFile.name;
		
		debug('Extracting... ', name);
		
		shapefilesDir = Template("$projectDirectory/shapefiles").substitute(projectDirectory=projectDirectory);
		
		commandline = Template("$terragearRoot/unzip -o -q $shapefilesDir/$name -d $shapefilesDir").substitute(terragearRoot=terragearRoot, shapefilesDir=shapefilesDir, name=name);
		
		try:
			subprocess.check_call(commandline);
		
			debug('Extracting done.');
		
			debug('Deleting zip file.');
		
			os.remove(Template("$shapefilesDir/$name.zip").substitute(shapefilesDir=shapefilesDir, name=name));
	
		except Exception, e:
			sucessfull = False;
			errors.append(repr(e));
	
	if (not sucessfull):
		raise DownloadShapesException(string.join(errors, "\n"));
	
	return;

def _shapeDecode(projectDirectory, configuration, terragearRoot):
	
	sucessfull = True;
	
	errors = [];
	
	for shapeFile in configuration.shapeFiles:
		name = shapeFile.name;

		shapefilesDir = Template("$projectDirectory/shapefiles").substitute(projectDirectory=projectDirectory);

		if (os.path.exists(shapefilesDir + "/" + name + ".shp")):
		
			shapeType = shapeFile.shapeType;
			
			commandline = None;
	
			material = shapeFile.material;
			
			if (isinstance(shapeType, Shape)):
				maxSegment = shapeType.maxSegmentLength;
				
				if (isinstance(shapeType, Line)):
					width = shapeType.width;		
					commandline = Template("$terragearRoot/shape-decode --line-width $width --max-segment $maxSegment $shapefilesDir/$name $projectDirectory/$material $material").substitute(terragearRoot=terragearRoot, width=width, maxSegment=maxSegment, shapefilesDir=shapefilesDir, name=name, projectDirectory=projectDirectory, material=material);
				else:
					commandline = Template("$terragearRoot/shape-decode --max-segment $maxSegment $shapefilesDir/$name $projectDirectory/$material $material").substitute(terragearRoot=terragearRoot, maxSegment=maxSegment, shapefilesDir=shapefilesDir, name=name, projectDirectory=projectDirectory, material=material);
			elif (isinstance(shapeType, Point)):
				size = shapeType.size;
				commandline = Template("$terragearRoot/shape-decode --point-width $size $shapefilesDir/$name $projectDirectory/$material $material").substitute(terragearRoot=terragearRoot, size=size ,shapefilesDir=shapefilesDir, name=name, projectDirectory=projectDirectory, material=material);
			
			try:
	
				debug('Decoding... ', name);
	
				error = subprocess.check_call(commandline);
				
				debug('Decoding done.');
				
			except CalledProcessError, e:
				sucessfull = False;
				errors.append(Template(SHAPE_DECODE_ERROR).safe_substitute(code = e.returncode));
			except Exception, e:
				sucessfull = False;
				errors.append(repr(e));
		else: 
			debug('Nothing to do, skpping:', name);
			
		debug('Deleting shape files...');
		
		files = fnmatch.filter(os.listdir(shapefilesDir), name + ".*")
		for file in files:
			os.remove(shapefilesDir + "/" + file);

		debug('Deleting shape files done.');

	
	if (not sucessfull):
		raise DownloadShapesException(string.join(errors, "\n"));
	
	return;

def processShapes(projectDirectory, configurationFile, terragearRoot):
	
	__checkArguments(projectDirectory, configurationFile, terragearRoot);

	configuration = __createConfigurationModel(configurationFile);
	
	_downloadShapes(projectDirectory, configuration);
	
	_unzipShapes(projectDirectory, configuration, terragearRoot);
	
	_shapeDecode(projectDirectory, configuration, terragearRoot);
	
	return;

if __name__ == '__main__':

	optionParser = OptionParser(usage=USAGE, version=VERSION);
	optionParser.add_option("-c", "--config", dest="configurationFile", help=CONFIG_OPTION_USAGE, metavar="FILE", default="./ShapeConfig.xml");
	optionParser.add_option("-t", "--terragear-root", dest="terragearRoot", help=TERRAGEAR_OPTION_USAGE, metavar="TERRAGEAR_ROOT", default="./Terragear");
	
	(options, arguments) = optionParser.parse_args();

	if (len(arguments) < 1):
		print MISSING_PROJECT;
		optionParser.print_help();

	projectDirectory = arguments[0];
	configurationFile = options.configurationFile;
	terragearRoot = options.terragearRoot;
	
	try:
		processShapes(projectDirectory, configurationFile, terragearRoot);
	except DownloadShapesException, e:
		print e.message;
